{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "3be9d3ad-67e9-4b0b-8c00-a02423ec36a5",
    "_uuid": "1ee3671ecaa16e57d5b85b222ea5240d58e158b9",
    "scrolled": true,
    "trusted": true
   },
   "outputs": [],
   "source": [
    "\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline\n",
    "from subprocess import check_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b4058c60-f3c6-4b69-b3a3-fadc5c11609b",
    "_uuid": "1b9011f635913ffc34f16e970fc7863c73b05150",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\"Data/train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "ac4e7a28-2d86-4652-9282-74d7fe054270",
    "_uuid": "3de12a3298a4ea9259e41f97b84ef31f90230d71",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_songs = pd.read_csv(\"Data/songs.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "256bbaef-f091-4881-8e09-ca76f7579b51",
    "_uuid": "1ffa39c2531092ff83ed1ba66c0eae596b929ce8",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_songs_extra = pd.read_csv(\"Data/song_extra_info.csv\")\n",
    "\n",
    "df_members = pd.read_csv(\"Data/members.csv\",parse_dates=[\"registration_init_time\",\"expiration_date\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "d7c1e080-4c06-42c6-aaef-a2aa7b5a4b20",
    "_uuid": "e0dcad8c5f7ecc0605b2e3c41ef7c9f039eb0712",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(\"Data/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "585859e2-8682-4596-905e-75ad529beb66",
    "_uuid": "e8bccb089f845cc9fc63a1b653c61ff1e206266e",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train =df_train.merge(df_songs,how=\"left\",on=\"song_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "cf973f60-d021-46e5-a48e-bcf8812fb2b2",
    "_uuid": "af3c466befdd422406896e5fcc9c101fdb427bdd",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train = df_train.merge(df_members,how=\"left\",on=\"msno\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "1089de98-9066-43d5-821b-5fea8f91bebd",
    "_uuid": "fc33204461c879b7e0ac2cfcd45b44e1b11af004",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_test =df_test.merge(df_songs,how=\"left\",on=\"song_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "7ffed4bf-3fa1-4ce3-90c0-8522cbc0a3fa",
    "_uuid": "213b228b285361340b36a72dff21c8ff006196b4",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_test = df_test.merge(df_members,how=\"left\",on=\"msno\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "e333409f-d770-4341-9bfe-601b9832f178",
    "_uuid": "127f06c90014eaeef9765d6d29802d35e01fe00b"
   },
   "source": [
    "**Handling Null Values **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b393f6d1-fe63-4b4e-9261-bdad96ea9fef",
    "_uuid": "cc6a3654aeb25cefc6388246374fc4dcae717d83",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['gender'].fillna(value=\"Unknown\",inplace=True)\n",
    "df_test['gender'].fillna(value=\"Unknown\",inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "ebe1fdab-486e-41ba-847b-493576c1ca97",
    "_uuid": "76cd4f59b94b950e19906401dd14efecf4bbcc48",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['source_system_tab'].fillna(value=\"Unknown\",inplace=True)\n",
    "df_test['source_system_tab'].fillna(value=\"Unknown\",inplace=True)\n",
    "\n",
    "\n",
    "df_train['source_screen_name'].fillna(value=\"Unknown\",inplace=True)\n",
    "df_test['source_screen_name'].fillna(value=\"Unknown\",inplace=True)\n",
    "\n",
    "df_train['source_type'].fillna(value=\"Unknown\",inplace=True)\n",
    "df_test['source_type'].fillna(value=\"Unknown\",inplace=True)\n",
    "\n",
    "df_train['genre_ids'].fillna(value=\"Unknown\",inplace=True)\n",
    "df_test['genre_ids'].fillna(value=\"Unknown\",inplace=True)\n",
    "\n",
    "df_train['artist_name'].fillna(value=\"Unknown\",inplace=True)\n",
    "df_test['artist_name'].fillna(value=\"Unknown\",inplace=True)\n",
    "\n",
    "df_train['composer'].fillna(value=\"Unknown\",inplace=True)\n",
    "df_test['composer'].fillna(value=\"Unknown\",inplace=True)\n",
    "\n",
    "df_train['lyricist'].fillna(value=\"Unknown\",inplace=True)\n",
    "df_test['lyricist'].fillna(value=\"Unknown\",inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "75a04a26-7c80-49fe-8d2a-b2e661f16cf5",
    "_uuid": "6bf2e0f16d8245cf836d0f4b6b7eb4b3b17ef4c0",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['song_length'].fillna(value=df_train['song_length'].mean(),inplace=True)\n",
    "\n",
    "df_test['song_length'].fillna(value=df_test['song_length'].mean(),inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "0d21fffb-1e85-4780-adc7-92af115af6ee",
    "_uuid": "cf040fb40ca0d23245ac64dc30f03440783f838f",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['language'].fillna(value=df_train['language'].mode()[0],inplace=True)\n",
    "\n",
    "df_test['language'].fillna(value=df_test['language'].mode()[0],inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "1f014333-f955-4981-a3ea-dde56af6dfa4",
    "_uuid": "898c7a025be2ac33db60645424436fb16deee0fd"
   },
   "source": [
    "**Genre IDS**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "43387f09-0422-48be-915b-0394583eae90",
    "_uuid": "972d48d02721a4c57e9fdfedba6bd98c111e8a7b",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['genre_ids'] = df_train['genre_ids'].str.split(\"|\")\n",
    "\n",
    "df_test['genre_ids'] = df_test['genre_ids'].str.split(\"|\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "821f9d07-0266-4014-9456-c4950a5e24b8",
    "_uuid": "dcbb6c6bb323c2c3bbf77924013ccde738b9663b",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['genre_count'] = df_train['genre_ids'].apply(lambda x : len(x) if \"Unknown\" not in x else 0)\n",
    "\n",
    "df_test['genre_count'] = df_test['genre_ids'].apply(lambda x : len(x) if \"Unknown\" not in x else 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "cd7e49ae-88d0-481f-bdc7-95b318c8b7fd",
    "_uuid": "c5f89ab485172aaf1d96b60e911c514685f7f4d0"
   },
   "source": [
    "**Artists **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "d59dcfad-2271-4f9f-9e5c-d97f25ae6774",
    "_uuid": "2c141b0fe803b1f506abd04dc9715046eb09ca1b",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['artist_name'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "eb9065f0-a15b-41b8-87d6-d07aff16bf70",
    "_uuid": "e68ffa35d8f9116652875920c08a19d79e2ac3a2",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_test['artist_name'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "4fbf7433-f2e1-4c1f-9f33-42bf035c287f",
    "_uuid": "dd0f6621b8fd55f05d97660e152368b388d8859b",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "print(\"Number of Common Artists in both train & Test : \", len(set.intersection(set(df_train['artist_name']), set(df_test['artist_name']))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "f920d26a-4ffd-45ed-8631-d27acc8d4b03",
    "_uuid": "2b7cfeb72f4af921f56593fed42ef0e5189ef607",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_artists = df_train.loc[:,[\"artist_name\",\"target\"]]\n",
    "\n",
    "artists1 = df_artists.groupby([\"artist_name\"],as_index=False).sum().rename(columns={\"target\":\"repeat_count\"})\n",
    "\n",
    "artists2 = df_artists.groupby([\"artist_name\"],as_index=False).count().rename(columns = {\"target\":\"play_count\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "4188eaaf-fa9d-4816-bf8b-afc2ea44e203",
    "_uuid": "8596e9cecd6bd23290eeeb0393038edafaf11eb8",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_artist_repeats = artists1.merge(artists2,how=\"inner\",on=\"artist_name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "82b39c07-69e1-4916-95b8-69f021d8c7fc",
    "_uuid": "35799dce25f165363d58bf5318c9fb4963daba67",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_artist_repeats.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "06440f93-2b89-474f-bdf7-8569e756e282",
    "_uuid": "e137f665bb7303f80bac435ff7c33f79844dc154",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_artist_repeats[\"repeat_percentage\"] = round((df_artist_repeats['repeat_count']*100) / df_artist_repeats['play_count'],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "8d8a3387-65b3-47ab-be7f-e38185ba7c3c",
    "_uuid": "277b8bf350dcf01fe92f9d9db79714d159778885",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_artist_repeats.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "cc9b240d-fcc5-4841-828a-791d3ee02175",
    "_uuid": "37463320e1e3d184cb73d832712f62ded6f90394",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_artist_repeats.drop([\"repeat_count\",\"play_count\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "714eb20c-f28f-47f4-865f-55e6cbc2f2e6",
    "_uuid": "e391157bd7ca80f932afc370fee6a37ffa79d854",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train = df_train.merge(df_artist_repeats,on=\"artist_name\",how=\"left\").rename(columns={\"repeat_percentage\":\"artist_repeat_percentage\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "22c23a1a-9e83-4db1-97d5-2aab05fe7e28",
    "_uuid": "039afb8614adad72c1faf64f6e6dc666c7acd4ae",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_test = df_test.merge(df_artist_repeats,on=\"artist_name\",how=\"left\").rename(columns={\"repeat_percentage\":\"artist_repeat_percentage\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "0176afab-13e4-4db3-89be-7afc73804939",
    "_uuid": "6eecce7928c791ec07532155b93e541621b7520a",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train.drop([\"genre_ids\",\"artist_name\"],axis=1,inplace=True)\n",
    "\n",
    "df_test.drop([\"genre_ids\",\"artist_name\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "57e24923-1462-4285-bdb4-ea1093318211",
    "_uuid": "cbb8504db58875de0c798d3c544f7ff1d1628815",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "del df_artist_repeats\n",
    "del df_artists"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "847f7903-0a7a-4fb9-8db1-45e38166f9fe",
    "_uuid": "d16cb1e66a425209a4dee8f9ab7c907b72a973e7"
   },
   "source": [
    "**Composers**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "c03fd148-a10a-45ae-a63a-38b59c73563f",
    "_uuid": "a0a6b63e3ea678e821b48254502d79d8214f9ff0",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['composer'] = df_train['composer'].str.split(\"|\") \n",
    "\n",
    "df_test['composer'] = df_test['composer'].str.split(\"|\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b6a6c3d6-448c-4405-a273-f4d467a2257c",
    "_uuid": "2e21eab9e5820879349f171ed3fa9eed5bfde6d8",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['composer_count'] = df_train['composer'].apply(lambda x : len(x) if \"Unknown\" not in x else 0 )\n",
    "\n",
    "df_test['composer_count'] = df_test['composer'].apply(lambda x : len(x) if \"Unknown\" not in x else 0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "2bd42a5a-065a-4602-bdce-48f4d0ad4ec2",
    "_uuid": "1f8603320468900ba689e556e8ccfce8d89a77a1",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['source_system_tab'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "5f83b683-bbd8-440f-b452-bb051e13ed85",
    "_uuid": "87207cc03d7e022d1f07a800b2c5d3d062ce0e3b",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "source_tab_dict = {\"my library\":8,\"discover\":7,\"search\":6,\"radio\":5,\"listen with\":4,\"explore\":3,\"notification\":2,\"settings\":1,\"Unknown\":0 }\n",
    "\n",
    "source_screen_name_dict = {\"Local playlist more\":19,\"Online playlist more\":18,\"Radio\":17,\"Unknown\":16,\"Album more\":15,\"Search\":14,\"Artist more\":13,\"Discover Feature\":12,\"Discover Chart\":11,\"Others profile more\":10,\"Discover Genre\":9,\"My library\":8,\"Explore\":7,\"Discover New\":6,\"Search Trends\":5,\"Search Home\":4,\"My library_Search\":3,\"Self profile more\":2,\"Concert\":1,\"Payment\":0}\n",
    "\n",
    "source_type_dict = {\"local-library\":12,\"online-playlist\":11,\"local-playlist\":10,\"radio\":9,\"album\":8,\"top-hits-for-artist\":7,\"song\":6,\"song-based-playlist\":5,\"listen-with\":4,\"Unknown\":3,\"topic-article-playlist\":2,\"artist\":1,\"my-daily-playlist\":0}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "f4e13a8d-62ee-41bb-8bde-9cf9ed5be6a2",
    "_uuid": "2dc0c85187e56c15fc2f2f55a6ec7c69994b6cb2",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['source_system_tab'] = df_train['source_system_tab'].map(source_tab_dict)\n",
    "\n",
    "df_test['source_system_tab'] = df_test['source_system_tab'].map(source_tab_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "8a8b9ee9-e8ae-4c97-9bb1-f27a216d3dc6",
    "_uuid": "3a2d33dc1d50072b100e0bedd0af6171f8a9d329",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['source_type'] = df_train['source_type'].map(source_type_dict)\n",
    "df_test['source_type'] = df_test['source_type'].map(source_type_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "f88437ea-8f9b-4459-abbf-63019285b2f4",
    "_uuid": "cb6789748cbf48b88faf327d9193f8c95f6845d2",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['source_screen_name'] = df_train['source_screen_name'].map(source_screen_name_dict)\n",
    "df_test['source_screen_name'] = df_test['source_screen_name'].map(source_screen_name_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "da84b5e0-043e-4327-b14a-e5311561ffe4",
    "_uuid": "8b3b0f82053eacf64f38f7bad0c6d1ba48909967",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "gender_train = pd.get_dummies(df_train['gender'],drop_first=True)\n",
    "\n",
    "gender_test = pd.get_dummies(df_test['gender'],drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "6fc1f6ab-5a90-4a85-b0f1-2e22ddeeef07",
    "_uuid": "7125f13a5e4599450dd9e0ed6bac1e757704e77d",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train = pd.concat([df_train,gender_train],axis=1)\n",
    "df_test = pd.concat([df_test,gender_test],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "febb5adf-734d-4319-a68c-bd81c9fa5db0",
    "_uuid": "9c2bcb89e3fb146650d18df05eefc9511963f2a9",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train.drop([\"composer\",\"gender\"],axis=1,inplace=True)\n",
    "df_test.drop([\"composer\",\"gender\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "c1624284-df7a-48ee-b718-9a42e2fef95b",
    "_uuid": "74441ca4fcc46337e8a45d4d6f4ae00b089d92ab",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['validity_days'] = (df_train['expiration_date'] - df_train['registration_init_time']).dt.days\n",
    "\n",
    "df_test['validity_days'] = (df_test['expiration_date'] - df_test['registration_init_time']).dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "9d39efaa-097a-4300-a7dc-7d1f4a045f35",
    "_uuid": "80afd39e48cd34339749a7a54d50093af6861d77",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train.drop([\"registration_init_time\",\"expiration_date\"],axis=1,inplace=True)\n",
    "\n",
    "df_test.drop([\"registration_init_time\",\"expiration_date\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "d10481d8-5d37-4b69-8a03-6e7a1cc113c0",
    "_uuid": "cf6160a23ad3a27db649418cf05aa7729d73a564",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['lyricist'] = df_train['lyricist'].str.split(\"|\")\n",
    "df_test['lyricist'] = df_test['lyricist'].str.split(\"|\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "f23ea795-cb58-437c-98df-7e650316b9df",
    "_uuid": "fea708201b7ee12138b1dad16446e540964c785f",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['lyricist_count'] = df_train['lyricist'].apply(lambda x : len(x) if \"Unknown\" not in x else 0 )\n",
    "\n",
    "df_test['lyricist_count'] = df_test['lyricist'].apply(lambda x : len(x) if \"Unknown\" not in x else 0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "cd80e974-13d4-4eeb-9be0-9ae6d1dfc11c",
    "_uuid": "2d88b1a9e657095201cc66e6399603c5701515fb",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_test['artist_repeat_percentage'].fillna(value=0.0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "228cee29-2e5e-4c90-b95a-daed1e4b8b76",
    "_uuid": "ea03009aafdf677f6b2f4223271415e38d223a6d",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_test['source_screen_name'].fillna(df_test['source_screen_name'].mode()[0],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "a10c3493-ed10-4997-884d-9b3334551d0a",
    "_uuid": "6aa69895f5e0d3beb080e989a235068b51058932",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train.drop(\"lyricist\",axis=1,inplace=True)\n",
    "\n",
    "df_test.drop(\"lyricist\",axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "f7822c81-3a8b-406e-a2d5-c4fd8625ab20",
    "_uuid": "8a1e918bcfbc8da8abbf9a01fb17b8a573fd22b0",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_songs_extra.drop(\"name\",axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "4c44e8d6-38d6-4dd4-9d26-7f2a3181eead",
    "_uuid": "16da8371863f89062eee31bc78b294514b783da7",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train = df_train.merge(df_songs_extra,how=\"left\",on=\"song_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "6d09a759-fd75-4265-a8e3-60e49a8cb1d5",
    "_uuid": "3360d854c88352e35276bdec0956faef525031a4",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_test = df_test.merge(df_songs_extra,how=\"left\",on=\"song_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "dd168c08-65cb-437c-b5b4-d494970a2507",
    "_uuid": "b64c279a1f045b3b9bb09a09bababaf5584385fe",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "def isrc_to_year(isrc):\n",
    "    if type(isrc) == str:\n",
    "        if int(isrc[5:7]) > 17:\n",
    "            return 1900 + int(isrc[5:7])\n",
    "        else:\n",
    "            return 2000 + int(isrc[5:7])\n",
    "    else:\n",
    "        return np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "9df350f0-b181-4c82-a2ca-5b277b45b2c4",
    "_uuid": "0977a4dc20a4247c6bd84f6b0511361beef964b1",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['song_year'] = df_train['isrc'].apply(isrc_to_year)\n",
    "\n",
    "df_test['song_year'] = df_test['isrc'].apply(isrc_to_year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "8e2beda3-d199-471c-bd55-f333bcf594be",
    "_uuid": "5db438f3fb88e57d11dd1b9b25dda2df2d74079a",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train.drop(\"isrc\",axis=1,inplace=True)\n",
    "df_test.drop(\"isrc\",axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "c1fd657a-0aa9-4116-b9a3-f499b554031e",
    "_uuid": "6ce71fb6b6d3ea809d8e6960bd0011500d14ecca",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['song_year'].fillna(value=-1,inplace=True)\n",
    "\n",
    "df_test['song_year'].fillna(value=-1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "fd44a946-989e-4146-b7ac-63f7138ebab5",
    "_uuid": "19fed010df11e9c8c25eace5c34d9c3ceb6234c8",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['song_year'] = df_train['song_year'].astype(\"int\")\n",
    "\n",
    "df_test['song_year'] = df_test['song_year'].astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "86970716-445f-46bf-9823-cf50fda4cc86",
    "_uuid": "67e18e841aa47b676c8b049efa84b3ce669955dc",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['source_system_tab'] = df_train['source_system_tab'].astype(\"category\")\n",
    "df_test['source_system_tab'] = df_test['source_system_tab'].astype(\"category\")\n",
    "\n",
    "df_train['source_screen_name'] = df_train['source_screen_name'].astype(\"category\")\n",
    "df_test['source_screen_name'] = df_test['source_screen_name'].astype(\"category\")\n",
    "\n",
    "df_train['source_type'] = df_train['source_type'].astype(\"category\")\n",
    "df_test['source_type'] = df_test['source_type'].astype(\"category\")\n",
    "\n",
    "df_train['language'] = df_train['language'].astype(\"category\")\n",
    "df_test['language'] = df_test['language'].astype(\"category\")\n",
    "\n",
    "df_train['city'] = df_train['city'].astype(\"category\")\n",
    "df_test['city'] = df_test['city'].astype(\"category\")\n",
    "\n",
    "\n",
    "df_train['registered_via'] = df_train['registered_via'].astype(\"category\")\n",
    "df_test['registered_via'] = df_test['registered_via'].astype(\"category\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "01440081-c704-43b2-8eef-894c92b4766e",
    "_uuid": "885dac82ccfc93a1840cda747adeb0dd03504f95",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train['age_range'] = pd.cut(df_train['bd'],bins=[-45,0,10,18,35,50,80,1052])\n",
    "df_test['age_range'] = pd.cut(df_test['bd'],bins=[-45,0,10,18,35,50,80,1052])\n",
    "\n",
    "combine = [df_train, df_test]\n",
    "\n",
    "for dataset in combine : \n",
    "    \n",
    "    dataset.loc[(dataset['bd'] > 0) & (dataset['bd'] <= 10), 'age_category'] = 0\n",
    "    dataset.loc[(dataset['bd'] > 80) & (dataset['bd'] <= 1052), 'age_category'] = 1\n",
    "    dataset.loc[(dataset['bd'] > 50) & (dataset['bd'] <= 80), 'age_category'] = 2\n",
    "    dataset.loc[(dataset['bd'] > 10) & (dataset['bd'] <= 18), 'age_category'] = 3\n",
    "    dataset.loc[(dataset['bd'] > 35) & (dataset['bd'] <= 50), 'age_category'] = 4\n",
    "    dataset.loc[(dataset['bd'] > -45) & (dataset['bd'] <= 0), 'age_category'] = 5\n",
    "    dataset.loc[(dataset['bd'] > 18) & (dataset['bd'] <= 35), 'age_category'] = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "ecb09faa-5f9d-46a2-ac66-bf89cbf91dc6",
    "_uuid": "e38c2360dfa72e6372941f9e82d77a94f3a36253",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "df_train.drop([\"age_range\",\"bd\"],axis=1,inplace=True)\n",
    "df_test.drop([\"age_range\",\"bd\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "41b41da0-d298-44b9-9a3a-8dc64697ea87",
    "_uuid": "5e0e61ad8cf7f9691d4558e1003cac7635501d7a",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "X = df_train.drop([\"msno\",\"song_id\",\"target\"],axis=1).values\n",
    "\n",
    "y = df_train['target'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "24bd0dd1-9394-4b33-aad8-c83c94b98273",
    "_uuid": "63d7df45004db1ca0cf17b23cccef904dd62beb3",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "\n",
    "d_train = lgb.Dataset(X, y)\n",
    "watchlist = [d_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "25941bef-eaf2-440a-9a55-954508fff472",
    "_uuid": "15ab51bca1f11ea37095059f724ea87aa7d75bfd",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "params = {}\n",
    "params['learning_rate'] = 0.5\n",
    "params['application'] = 'binary'\n",
    "params['max_depth'] = 10\n",
    "params['num_leaves'] = 2**6\n",
    "params['verbosity'] = 0\n",
    "params['metric'] = 'auc'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "1ea1427e-ef46-4800-9d8d-79497ccd9cdc",
    "_uuid": "541cd95af613594809a2a54d15471ca260acfef1",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "model = lgb.train(params, train_set=d_train, num_boost_round=60, valid_sets=watchlist, \\\n",
    "verbose_eval=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "3512aff3-192f-4ba8-a9b6-49b221c5a61a",
    "_uuid": "7ee87272a9802079d94efb64a6f704b217ada478",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "song_ids = df_test['id'].values\n",
    "\n",
    "X_test = df_test.drop([\"msno\",\"song_id\",\"id\"],axis=1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "815a12c1-6ec6-4790-83b7-c0e24b364873",
    "_uuid": "8214a9fd260d2c080bcab4224c85f4bdc2183dcc",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "y_preds = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "530d4069-d8d1-45d4-a032-4034c170b101",
    "_uuid": "28c03f87d8c95a15965bf60453af0040a9787559",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "result_df = pd.DataFrame()\n",
    "\n",
    "result_df['id'] = song_ids\n",
    "result_df['target'] = y_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "3108e46f-2e64-4adb-8339-92a79babec24",
    "_uuid": "14d6936aac0d2770382a00ba31d25fe0a876906c",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    "result_df.to_csv('submission_new.csv.gz', compression = 'gzip', index=False, float_format = '%.5f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "e7d9c757-1758-475d-adc4-1649d1912394",
    "_uuid": "e689ed459e87acdc172416e98568f8993d9bdfe9",
    "collapsed": true,
    "trusted": false
   },
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "pygments_lexer": "ipython3",
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "version": "3.6.3",
   "nbconvert_exporter": "python",
   "name": "python",
   "codemirror_mode": {
    "version": 3,
    "name": "ipython"
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
